﻿using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Text;

namespace MyConsoleApp.Code
{
    class HtmlAgilityPack
    {

        /*            
         * string html = new Program().HttpGet("http://www.oschina.net/");
            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(html);
            //获取文章列表
            var artlist = doc.DocumentNode.SelectNodes("//ul[@class='p1']");
            foreach (var item in artlist)
            {
                HtmlDocument adoc = new HtmlDocument();
                adoc.LoadHtml(item.InnerHtml);
                var html_alist = adoc.DocumentNode.SelectNodes("//a[@target='_blank']");
                foreach (var html_a in html_alist)
                    Console.WriteLine(string.Format("标题为：{0}，链接为：{1}", html_a.InnerText, html_a.Attributes["href"].Value));
            }
         */
        public string HttpGet(string url)
        {
            string responsestr = "";
            HttpWebRequest req = HttpWebRequest.Create(url) as HttpWebRequest;
            req.Accept = "text/html, application/xhtml+xml, */*";
            req.Method = "GET";
            req.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko";
            using (HttpWebResponse response = req.GetResponse() as HttpWebResponse)
            {
                Stream stream;
                if (response.ContentEncoding.ToLower().Contains("gzip"))
                {
                    stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress);
                }
                else if (response.ContentEncoding.ToLower().Contains("deflate"))
                {
                    stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress);
                }
                else
                {
                    stream = response.GetResponseStream();
                }
                using (StreamReader reader = new StreamReader(stream, GetEncoding(response.CharacterSet)))
                {
                    responsestr = reader.ReadToEnd();
                    stream.Dispose();
                }
            }
            return responsestr;
        }

        public Encoding GetEncoding(string CharacterSet)
        {
            switch (CharacterSet.ToLower())
            {
                case "gb2312": return Encoding.GetEncoding("gb2312");
                case "utf-8": return Encoding.UTF8;
                default: return Encoding.Default;
            }
        }
    }
}
